#!/bin/bash

#Created by cheng 2016-05-23
#Project Apollo V1 SdsHttpStatH


echo `date +"%Y-%m-%d %H:%M:%S"`  "------------------------------------" | tee -a $LOGFILE
echo `date +"%Y-%m-%d %H:%M:%S"`  "-----------Project Apollo V1 SdsHttpStatH--------" | tee -a $LOGFILE
echo `date +"%Y-%m-%d %H:%M:%S"`  "Shell Version 1.0 update(2016-05-23)" | tee -a $LOGFILE
echo `date +"%Y-%m-%d %H:%M:%S"`  "------------------------------------" | tee -a $LOGFILE

if [ $# -lt 2 ]; then
        echo `date +"%Y-%m-%d %H:%M:%S"`  "Usage: <IDCNO> <ZOOKEEPER>"
        exit 1
fi

IDCNO=$1
ZK=$2
WORKPATH=/usr/local/storm10
JAVALIB=${WORKPATH}/topolib/
DATE=`date -d "1 day ago" +%Y%m%d`
LOGFILE=/home/spark/logs/apollo_SdsHttpStatH_${DATE}.log
TABLENAME=SDS_HTTP_STAT_H

n=0
while(( $n <= 23 ))
do
        HOUR=$(printf "%02d\n" $n)
        INPUTPATH=/user/hive/warehouse/idc_access_log/${IDCNO}/${DATE}/${HOUR}
        OUTPUTPATH=/user/hive/warehouse/apo_sdshttpstat_h/${DATE}/${HOUR}
echo "${INPUTPATH} ${OUTPUTPATH} ${HOUR} ${ZK} " |tee -a $LOGFILE
        spark-submit --class com.aotain.project.apollo.spark.SdsHttpStatH --deploy-mode cluster --master yarn --num-executors 170 --driver-memory 6g --executor-memory 6g --executor-cores 4 --conf "spark.core.connection.ack.wait.timeout=300" --conf "spark.default.parallelism=1320" --conf "spark.shuffle.consolidateFiles=true" --conf "spark.shuffle.memoryFraction=0.5" --conf "spark.yarn.executor.memoryOverhead=2048" --conf "spark.storage.memoryFraction=0.2" --conf spark.driver.extraJavaOptions="-XX:PermSize=128M -XX:MaxPermSize=256M" --conf "spark.reducer.maxMbInFlight=12" --conf "spark.akka.frameSize=20" --conf "spark.executor.extraClassPath=/opt/cloudera/parcels/CDH/lib/hbase/lib/htrace-core-3.1.0-incubating.jar" --driver-class-path "/opt/cloudera/parcels/CDH/lib/hbase/lib/htrace-core-3.1.0-incubating.jar"  --queue thequeue ${JAVALIB}/hades-project-0.0.1-SNAPSHOT.jar ${INPUTPATH} ${OUTPUTPATH} ${HOUR} ${ZK} > $LOGFILE 2>&1
        hive -e "use sds;alter table $TABLENAME add partition(date='$DATE',hour='$HOUR') location '$OUTPUTPATH';" 1>>$LOGFILE 2>>$LOGFILE

 (( n++ ))
done



echo `date +"%Y-%m-%d %H:%M:%S"`      "Exec Success [$DATE]!" | tee -a $LOGFILE

exit 0